#This file processes the langugage in given posts so that it is more manageable.

import WordBag
import re

#A list of the top 100 most common english works.
#We should probably find a larger list and store it as an external file.
#We could then have a control variable that lets us choose the number of top words that should be removed.
mostCommonEnglishWords = set(["the","be","to","of","and","a","in","that","have","i","it","for","not","on","with","he","as","you","do","at","this","but","his","by","from","they","we","say","her","she","or","an","will","my","one","all","would","there","their","what","so","up","out","if","about","who","get","which","go","me","when","make","can","like","time","no","just","him","know","take","people","into","year","your","good","some","could","them","see","other","than","then","now","look","only","come","its","over","think","also","back","after","use","two","how","our","work","first","well","way","even","new","want","because","any","these","give","day","most","us"])

#Features that can be turned on an off
class Features:
	#Indicates whether or not upper case and lower case correspond to different words
	CASE_SENSETIVE = False
	#Indicates whether we should count a word every time it occurs in a single post or just once
	COUNT_DUPLICATE_WORDS_IN_SAME_POST = False
	#Indicates wheterh or not we should remove the most common english words
	REMOVE_MOST_COMMON_ENGLISH_WORDS = True

#Splits the text on certain punctuation and line breaks
def splitText(text):
	return re.split(r'[ ."?!;,\n\r\t]+',text)

#Given a word list, applies each of the enabled features to narrow it down.
def processWords(words):
	finalWordList = []
	for word in words:
		if(not Features.CASE_SENSETIVE):
			word = word.lower()
		if(not (word == "")):
			if(Features.REMOVE_MOST_COMMON_ENGLISH_WORDS):
				if(not (word in mostCommonEnglishWords)):
					finalWordList.append(word)
			else:
				finalWordList.append(word)
	if(Features.COUNT_DUPLICATE_WORDS_IN_SAME_POST):
		return finalWordList
	else:
		return set(finalWordList)

#Splits the text into a word list and then applies the word processing featues
def processText(text):
	return processWords(splitText(text))

#Fills a given wordbag object with the processed words from the body of a parsed post object 
def fillWordBag(post, wordBag):	
	for word in processText(post.bodyMarkdown):
		wordBag.add(word, post.openStatus)